{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Vd1jTqs6eL9r"
   },
   "outputs": [],
   "source": [
    "# Install conda environment\n",
    "!pip install -q condacolab\n",
    "import condacolab\n",
    "condacolab.install()\n",
    "\n",
    "# Create and activate a new conda environment\n",
    "!conda create -n minigptv python=3.9 -y\n",
    "!conda activate minigptv\n",
    "\n",
    "# Install cudatoolkit\n",
    "!conda install -c anaconda cudatoolkit -y\n",
    "\n",
    "# Install other dependencies using pip\n",
    "!pip install torch==2.0.0 torchaudio torchvision huggingface-hub==0.18.0 matplotlib==3.7.0 psutil==5.9.4 iopath pyyaml==6.0 regex==2022.10.31 tokenizers==0.13.2 tqdm==4.64.1 transformers==4.30.0 timm==0.6.13 webdataset==0.2.48 omegaconf==2.3.0 opencv-python==4.7.0.72 decord==0.6.0 peft==0.2.0 sentence-transformers gradio==3.47.1 accelerate==0.20.3 bitsandbytes==0.37.0 scikit-image visual-genome wandb ipykernel\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 624882,
     "status": "ok",
     "timestamp": 1720681803363,
     "user": {
      "displayName": "linrui xu",
      "userId": "17232213947858556138"
     },
     "user_tz": -480
    },
    "id": "nBAc2IpUfA0z",
    "outputId": "74d40fbb-7442-4b1e-f402-bf16c3f5685b"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Cloning into 'MiniGPT-4-20240710'...\n",
      "remote: Enumerating objects: 1797, done.\u001B[K\n",
      "remote: Counting objects: 100% (884/884), done.\u001B[K\n",
      "remote: Compressing objects: 100% (227/227), done.\u001B[K\n",
      "remote: Total 1797 (delta 719), reused 657 (delta 657), pack-reused 913\u001B[K\n",
      "Receiving objects: 100% (1797/1797), 65.21 MiB | 11.79 MiB/s, done.\n",
      "Resolving deltas: 100% (1047/1047), done.\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Downloading...\n",
      "From (original): https://drive.google.com/uc?id=11nAPjEok8eAGGEG1N2vXo3kBLCg0WgUk\n",
      "From (redirected): https://drive.google.com/uc?id=11nAPjEok8eAGGEG1N2vXo3kBLCg0WgUk&confirm=t&uuid=13709e67-eb58-4134-b23d-8cd47a0860c0\n",
      "To: /content/pretrained_minigpt4_llama2_7b.pth\n",
      "100%|██████████| 277M/277M [00:09<00:00, 27.9MB/s]\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Collecting modelscope\n",
      "  Downloading modelscope-1.16.0-py3-none-any.whl.metadata (44 kB)\n",
      "\u001B[?25l     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.0/44.2 kB\u001B[0m \u001B[31m?\u001B[0m eta \u001B[36m-:--:--\u001B[0m\r\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m44.2/44.2 kB\u001B[0m \u001B[31m4.8 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: sentencepiece in /usr/local/lib/python3.10/site-packages (0.2.0)\n",
      "Requirement already satisfied: requests>=2.25 in /usr/local/lib/python3.10/site-packages (from modelscope) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.64.0 in /usr/local/lib/python3.10/site-packages (from modelscope) (4.64.1)\n",
      "Requirement already satisfied: urllib3>=1.26 in /usr/local/lib/python3.10/site-packages (from modelscope) (2.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests>=2.25->modelscope) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests>=2.25->modelscope) (3.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests>=2.25->modelscope) (2023.11.17)\n",
      "Downloading modelscope-1.16.0-py3-none-any.whl (5.7 MB)\n",
      "\u001B[2K   \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m5.7/5.7 MB\u001B[0m \u001B[31m15.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hInstalling collected packages: modelscope\n",
      "Successfully installed modelscope-1.16.0\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Downloading: 100%|██████████| 614/614 [00:00<00:00, 1.38kB/s]\n",
      "Downloading: 0.00B [00:00, ?B/s]\n",
      "Downloading: 100%|██████████| 188/188 [00:00<00:00, 337B/s]\n",
      "Downloading: 100%|██████████| 6.86k/6.86k [00:00<00:00, 13.2kB/s]\n",
      "Downloading: 100%|██████████| 9.29G/9.29G [02:26<00:00, 68.1MB/s]\n",
      "Downloading: 100%|██████████| 3.26G/3.26G [00:55<00:00, 62.8MB/s]\n",
      "Downloading: 100%|██████████| 26.2k/26.2k [00:00<00:00, 42.8kB/s]\n",
      "Downloading: 100%|██████████| 9.29G/9.29G [02:54<00:00, 57.2MB/s]\n",
      "Downloading: 100%|██████████| 3.26G/3.26G [00:56<00:00, 61.6MB/s]\n",
      "Downloading: 100%|██████████| 26.2k/26.2k [00:00<00:00, 45.3kB/s]\n",
      "Downloading: 100%|██████████| 10.2k/10.2k [00:00<00:00, 23.3kB/s]\n",
      "Downloading: 100%|██████████| 414/414 [00:00<00:00, 935B/s]\n",
      "Downloading: 100%|██████████| 1.76M/1.76M [00:01<00:00, 1.50MB/s]\n",
      "Downloading: 100%|██████████| 488k/488k [00:01<00:00, 410kB/s]\n",
      "Downloading: 100%|██████████| 1.58k/1.58k [00:00<00:00, 3.46kB/s]\n",
      "Downloading: 100%|██████████| 4.65k/4.65k [00:00<00:00, 10.5kB/s]\n"
     ]
    }
   ],
   "source": [
    "# Clone the repository and switch directory\n",
    "!git clone https://github.com/Czi24/MiniGPT-4-20240710.git\n",
    "\n",
    "# https://drive.google.com/file/d/1RY9jV0dyqLX-o38LrumkKRh6Jtaop58R/view?usp=sharing\n",
    "# https://drive.google.com/file/d/11nAPjEok8eAGGEG1N2vXo3kBLCg0WgUk/view?usp=sharing\n",
    "\n",
    "import gdown\n",
    "\n",
    "# Downloading files with gdown\n",
    "gdown.download(\"https://drive.google.com/uc?id=11nAPjEok8eAGGEG1N2vXo3kBLCg0WgUk\", output=None, quiet=False)\n",
    "\n",
    "# Download model file\n",
    "# Method 1\n",
    "# from huggingface_hub import snapshot_download\n",
    "# snapshot_download(repo_id=\"Vision-CAIR/vicuna-7b\", local_dir=\"vicuna-7b\")\n",
    "\n",
    "# Method 2\n",
    "# !git clone https://huggingface.co/Vision-CAIR/vicuna-7b\n",
    "\n",
    "# Method 3\n",
    "!pip install modelscope sentencepiece -U\n",
    "import torch\n",
    "from modelscope import snapshot_download\n",
    "model_dir = snapshot_download('shakechen/Llama-2-7b-chat-hf', cache_dir='./', revision='master')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 564,
     "status": "ok",
     "timestamp": 1720682175171,
     "user": {
      "displayName": "linrui xu",
      "userId": "17232213947858556138"
     },
     "user_tz": -480
    },
    "id": "5wOF6cAFgOXB",
    "outputId": "a3d4a505-5e41-4750-b30a-44d86f1d1f89"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "/content/MiniGPT-4-20240710\n",
      "CODE_OF_CONDUCT.md  environment.yml  \u001B[0m\u001B[01;34mexamples_v2\u001B[0m/      \u001B[01;34mminigpt4\u001B[0m/           \u001B[01;34mprompts\u001B[0m/        train.py\n",
      "\u001B[01;34mdataset\u001B[0m/            \u001B[01;34meval_configs\u001B[0m/    \u001B[01;34mfigs\u001B[0m/             MiniGPT4_Train.md   README.md\n",
      "demo.py             \u001B[01;34meval_scripts\u001B[0m/    LICENSE_Lavis.md  MiniGPTv2.pdf       SECURITY.md\n",
      "demo_v2.py          \u001B[01;34mexamples\u001B[0m/        LICENSE.md        MiniGPTv2_Train.md  \u001B[01;34mtrain_configs\u001B[0m/\n",
      "/content/MiniGPT-4-20240710\n"
     ]
    }
   ],
   "source": [
    "# Switch to the cloned repository directory\n",
    "%cd MiniGPT-4-20240710\n",
    "%ls\n",
    "\n",
    "# View the current working directory\n",
    "!pwd\n",
    "# %cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 909,
     "status": "ok",
     "timestamp": 1720682181299,
     "user": {
      "displayName": "linrui xu",
      "userId": "17232213947858556138"
     },
     "user_tz": -480
    },
    "id": "aiOCUWOzgVOF",
    "outputId": "3bb801bf-70e1-4b52-9619-e98e44be1d2f"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "model:\n",
      "  arch: minigpt4\n",
      "\n",
      "  # vit encoder\n",
      "  image_size: 224\n",
      "  drop_path_rate: 0\n",
      "  use_grad_checkpoint: False\n",
      "  vit_precision: \"fp16\"\n",
      "  freeze_vit: True\n",
      "  has_qformer: False\n",
      "\n",
      "  # generation configs\n",
      "  prompt: \"\"\n",
      "\n",
      "  llama_model: \"/content/shakechen/Llama-2-7b-chat-hf\"\n",
      "\n",
      "preprocess:\n",
      "    vis_processor:\n",
      "        train:\n",
      "          name: \"blip2_image_train\"\n",
      "          image_size: 224\n",
      "        eval:\n",
      "          name: \"blip2_image_eval\"\n",
      "          image_size: 224\n",
      "    text_processor:\n",
      "        train:\n",
      "          name: \"blip_caption\"\n",
      "        eval:\n",
      "          name: \"blip_caption\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Specify the configuration file path\n",
    "config_file_path = 'minigpt4/configs/models/minigpt4_llama2.yaml'\n",
    "new_llama_model_path = '/content/shakechen/Llama-2-7b-chat-hf'\n",
    "\n",
    "# Read an existing configuration file\n",
    "with open(config_file_path, 'r') as file:\n",
    "    lines = file.readlines()\n",
    "\n",
    "# Modify the llama_model path\n",
    "with open(config_file_path, 'w') as file:\n",
    "    for line in lines:\n",
    "        if line.strip().startswith('llama_model:'):\n",
    "            file.write(f'  llama_model: \"{new_llama_model_path}\"\\n')\n",
    "        else:\n",
    "            file.write(line)\n",
    "\n",
    "# Verify changes\n",
    "with open(config_file_path, 'r') as file:\n",
    "    updated_config = file.read()\n",
    "    print(updated_config)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1127,
     "status": "ok",
     "timestamp": 1720682191029,
     "user": {
      "displayName": "linrui xu",
      "userId": "17232213947858556138"
     },
     "user_tz": -480
    },
    "id": "Fe0CWZuQgWWw",
    "outputId": "673c2beb-970b-44e1-d5c6-a3aba2a8d90f"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "model:\n",
      "  arch: minigpt4\n",
      "  model_type: pretrain_llama2\n",
      "  max_txt_len: 160\n",
      "  end_sym: \"</s>\"\n",
      "  low_resource: True\n",
      "  prompt_template: '[INST] {} [/INST] '\n",
      "  ckpt: '/content/pretrained_minigpt4_llama2_7b.pth'\n",
      "\n",
      "\n",
      "datasets:\n",
      "  cc_sbu_align:\n",
      "    vis_processor:\n",
      "      train:\n",
      "        name: \"blip2_image_eval\"\n",
      "        image_size: 224\n",
      "    text_processor:\n",
      "      train:\n",
      "        name: \"blip_caption\"\n",
      "\n",
      "run:\n",
      "  task: image_text_pretrain\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Specify the configuration file path\n",
    "config_file_path = 'eval_configs/minigpt4_llama2_eval.yaml'\n",
    "new_llama_model_path = '/content/pretrained_minigpt4_llama2_7b.pth'\n",
    "\n",
    "# Read an existing configuration file\n",
    "with open(config_file_path, 'r') as file:\n",
    "    lines = file.readlines()\n",
    "\n",
    "# Modify the llama_model path\n",
    "with open(config_file_path, 'w') as file:\n",
    "    for line in lines:\n",
    "        if line.strip().startswith('ckpt:'):\n",
    "            file.write(f\"  ckpt: '{new_llama_model_path}'\\n\")\n",
    "        else:\n",
    "            file.write(line)\n",
    "\n",
    "# Verify changes\n",
    "with open(config_file_path, 'r') as file:\n",
    "    updated_config = file.read()\n",
    "    print(updated_config)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "Rl6qM5UtgZZD",
    "outputId": "107bf9a8-6760-4791-fffd-56068499c87c"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "Initializing Chat\n",
      "Loading checkpoint shards: 100% 2/2 [00:46<00:00, 23.24s/it]\n",
      "100% 1.89G/1.89G [01:27<00:00, 23.1MB/s]\n",
      "Do not use Q-Former here.\n",
      "Load MiniGPT-4 Checkpoint: /content/pretrained_minigpt4_llama2_7b.pth\n",
      "Initialization Finished\n",
      "/content/MiniGPT-4-20240710/demo.py:171: GradioDeprecationWarning: The `enable_queue` parameter has been deprecated. Please use the `.queue()` method instead.\n",
      "  demo.launch(share=True, enable_queue=True)\n",
      "Running on local URL:  http://127.0.0.1:7860\n",
      "IMPORTANT: You are using gradio version 3.47.1, however version 4.29.0 is available, please upgrade.\n",
      "--------\n",
      "Running on public URL: https://c771d12717225749c7.gradio.live\n",
      "\n",
      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n",
      "/usr/local/lib/python3.10/site-packages/gradio/helpers.py:818: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Textbox(...)` instead of `return gr.update(...)\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/gradio/components/image.py:193: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Image(...)` instead of `return gr.Image.update(...)`.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/gradio/components/textbox.py:163: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Textbox(...)` instead of `return gr.Textbox.update(...)`.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/gradio/components/button.py:89: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Button(...)` instead of `return gr.Button.update(...)`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "!python demo.py --cfg-path eval_configs/minigpt4_llama2_eval.yaml  --gpu-id 0"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [],
   "machine_shape": "hm",
   "authorship_tag": "ABX9TyM1EM00sU9Yub+bHkMQNcKH"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
